{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**NOTE:** This notebook should be the first one to be run in the analysis pipeline, since it defines the outliers that will be discarded in all future analyses."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports\n",
    "\n",
    "Import all the modules and functionalities we need."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:11.128510Z",
     "iopub.status.busy": "2021-09-10T00:52:11.128510Z",
     "iopub.status.idle": "2021-09-10T00:52:12.230611Z",
     "shell.execute_reply": "2021-09-10T00:52:12.229593Z",
     "shell.execute_reply.started": "2021-09-10T00:52:11.128510Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "# Import standard libraries.\n",
    "from collections import defaultdict\n",
    "import os\n",
    "\n",
    "# Import third party libraries.\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Import custom libraries/scripts.\n",
    "import annotations\n",
    "import helpers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading data\n",
    "\n",
    "Fetch all our relevant data for the current analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:12.232594Z",
     "iopub.status.busy": "2021-09-10T00:52:12.231610Z",
     "iopub.status.idle": "2021-09-10T00:52:12.246587Z",
     "shell.execute_reply": "2021-09-10T00:52:12.245586Z",
     "shell.execute_reply.started": "2021-09-10T00:52:12.232594Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 999 µs\n"
     ]
    }
   ],
   "source": [
    "# Set working contants.\n",
    "EXPERIMENTS_PATH = r'E:\\Miguel\\PhD\\Results\\Competition\\DL\\Four-Arena Setup\\WT_mating_status\\processed'\n",
    "FPS = 60\n",
    "N_MINUTES = 60\n",
    "N_FRAMES = N_MINUTES * 60 * FPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:12.250584Z",
     "iopub.status.busy": "2021-09-10T00:52:12.249586Z",
     "iopub.status.idle": "2021-09-10T00:52:12.261592Z",
     "shell.execute_reply": "2021-09-10T00:52:12.260592Z",
     "shell.execute_reply.started": "2021-09-10T00:52:12.250584Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Paths to conditions:\n",
      " ['F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added', 'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h', 'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h', 'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin']\n",
      "time: 999 µs\n"
     ]
    }
   ],
   "source": [
    "# Set the conditions to analyze.\n",
    "condition_order = ['virgin_virgin', 'virgin_mated24h', 'virgin_mated2h', 'virgin_added']\n",
    "conditions = [item.path for item in os.scandir(EXPERIMENTS_PATH) if item.name in condition_order]\n",
    "print('Paths to conditions:\\n', conditions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:12.263579Z",
     "iopub.status.busy": "2021-09-10T00:52:12.262591Z",
     "iopub.status.idle": "2021-09-10T00:52:12.309569Z",
     "shell.execute_reply": "2021-09-10T00:52:12.308599Z",
     "shell.execute_reply.started": "2021-09-10T00:52:12.263579Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['video_2017-09-07T13_28_02_arena2_slice1', 'video_2017-09-27T15_06_20_arena1_slice1', 'video_2017-09-28T14_44_01_arena1_slice1', 'video_2017-10-19T14_58_20_arena2_slice1', 'video_2017-10-26T14_43_16_arena1_slice1', 'video_2017-09-05T14_34_55_arena4', 'video_2017-09-12T14_36_55_arena4', 'video_2017-09-14T13_34_41_arena2', 'video_2017-09-21T14_47_10_arena3', 'video_2017-09-21T14_47_10_arena4', 'video_2017-09-26T13_40_19_arena4', 'video_2017-09-27T16_21_07_arena1', 'video_2017-09-27T16_21_07_arena3', 'video_2017-09-28T15_55_49_arena2', 'video_2017-10-12T14_34_41_arena2', 'video_2017-10-18T14_05_05_arena3', 'video_2017-10-19T14_58_20_arena1', 'video_2017-10-24T14_48_34_arena1', 'video_2017-10-24T14_48_34_arena4', 'video_2017-10-25T13_39_35_arena4', 'video_2017-11-03T14_54_45_arena2', 'video_2017-09-05T15_42_04_arena1', 'video_2017-11-02T16_11_06_arena1', 'video_2017-12-19T16_23_33_arena4', 'video_2017-09-05T15_42_04_arena3', 'video_2017-09-07T13_28_02_arena4']\n",
      "26\n",
      "time: 37 ms\n"
     ]
    }
   ],
   "source": [
    "# Create a list of all the experiments that we DO NOT want to analyze.\n",
    "intruders = []\n",
    "for condition_path in conditions:\n",
    "    condition = os.path.basename(condition_path)\n",
    "    for file in os.listdir(condition_path):\n",
    "        if 'quality_control' in file:\n",
    "\n",
    "            # Read the quality.csv file to check which experiments are usable for analysis.\n",
    "            quality_file = os.path.join(condition_path, file)\n",
    "            quality_df = pd.read_csv(quality_file, usecols=[0,1], index_col=0)\n",
    "            quality_df = quality_df[quality_df['is_usable'] == False].index.values\n",
    "\n",
    "            for value in quality_df:\n",
    "                intruders.append(value + '_slice1' if condition=='virgin_added' else value)\n",
    "\n",
    "print(intruders)             \n",
    "print(len(intruders))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:12.311549Z",
     "iopub.status.busy": "2021-09-10T00:52:12.310566Z",
     "iopub.status.idle": "2021-09-10T00:52:12.372529Z",
     "shell.execute_reply": "2021-09-10T00:52:12.371529Z",
     "shell.execute_reply.started": "2021-09-10T00:52:12.311549Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\n",
      " virgin_added\n",
      "\t\n",
      " virgin_mated24h\n",
      "video_2017-09-20T14_48_55_arena3\n",
      "\t\n",
      " virgin_mated2h\n",
      "\t\n",
      " virgin_virgin\n",
      "video_2017-09-06T13_42_13_arena3\n",
      "video_2017-09-26T14_52_18_arena3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'virgin_virgin': ['F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-05T13_28_41_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-05T14_34_55_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-06T13_42_13_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-06T14_45_32_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-07T14_37_38_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-07T14_37_38_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-08T13_26_13_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-12T13_30_06_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-12T14_36_55_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-12T16_18_15_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-13T14_34_18_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-20T13_43_53_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-20T14_48_55_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-21T13_38_55_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-21T15_57_23_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-26T13_40_19_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-28T13_36_06_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-28T14_44_01_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-09-28T15_55_49_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-12T15_50_43_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-13T16_06_33_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-18T15_19_17_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-19T13_50_56_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-20T13_21_06_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-20T15_44_09_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-24T16_13_35_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-10-25T14_55_12_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-11-02T13_37_48_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-11-02T14_48_28_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-11-02T16_11_06_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-11-03T14_54_45_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-11-03T16_20_50_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-12-20T12_58_51_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_virgin\\\\video_2017-12-20T14_01_01_arena3'],\n",
       " 'virgin_mated24h': ['F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-05T13_28_41_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-07T16_01_26_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-08T14_31_59_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-08T14_31_59_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-12T13_30_06_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-13T15_40_03_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-14T14_47_16_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-20T13_43_53_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-26T14_52_18_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-09-27T15_06_20_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-11T13_30_05_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-11T16_00_50_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-13T14_37_35_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-13T14_37_35_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-13T14_37_35_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-19T13_50_56_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-19T14_58_20_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-19T16_19_37_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-20T14_31_24_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-20T14_31_24_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-24T13_43_26_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-25T13_39_35_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-25T14_55_12_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-25T14_55_12_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-26T13_35_36_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-10-26T14_43_16_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-11-03T13_38_06_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-11-03T13_38_06_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-11-03T13_38_06_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated24h\\\\video_2017-11-03T16_20_50_arena3'],\n",
       " 'virgin_mated2h': ['F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-06T15_53_34_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-07T16_01_26_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-07T16_01_26_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-08T15_42_18_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-08T15_42_18_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-13T15_40_03_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-14T15_53_40_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-20T16_07_55_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-21T15_57_23_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-21T15_57_23_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-26T16_06_31_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-26T16_06_31_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-09-28T15_55_49_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-11T16_00_50_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-11T16_00_50_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-11T16_00_50_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-12T15_50_43_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-13T16_06_33_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-18T16_37_02_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-18T16_37_02_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-19T16_19_37_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-19T16_19_37_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-25T16_23_26_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-10-25T16_23_26_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-11-02T16_11_06_arena4',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-12-19T15_13_55_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-12-19T15_13_55_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-12-19T16_23_33_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-12-19T16_23_33_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-12-19T16_23_33_arena3',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_mated2h\\\\video_2017-12-20T15_08_53_arena2'],\n",
       " 'virgin_added': ['F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-05T13_28_41_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-05T13_28_41_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-06T13_42_13_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-06T14_45_32_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-06T14_45_32_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-08T13_26_13_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-08T13_26_13_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-12T14_36_55_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-12T16_18_15_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-12T16_18_15_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-13T13_31_57_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-20T16_07_55_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-20T16_07_55_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-21T13_38_55_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-21T14_47_10_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-09-26T16_06_31_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-11T14_44_35_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-11T14_44_35_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-12T13_31_34_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-13T13_32_39_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-18T16_37_02_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-18T16_37_02_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-19T13_50_56_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-20T14_31_24_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-20T14_31_24_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-26T14_43_16_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-26T16_04_34_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-10-26T16_04_34_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-11-02T14_48_28_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-11-02T14_48_28_arena2',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-12-20T12_58_51_arena1',\n",
       "  'F:\\\\Miguel\\\\PhD\\\\Results\\\\Competition\\\\DL\\\\Four-Arena Setup\\\\WT_mating_status\\\\processed\\\\virgin_added\\\\video_2017-12-20T14_01_01_arena2']}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 47 ms\n"
     ]
    }
   ],
   "source": [
    "# Load all usable experiments for each condition.\n",
    "experiments = {condition: [] for condition in condition_order}\n",
    "\n",
    "for condition_path in conditions:\n",
    "    \n",
    "    condition = os.path.basename(condition_path)\n",
    "    print('\\t\\n', condition)\n",
    "    \n",
    "    for item in os.scandir(condition_path):\n",
    "        if item.is_dir() and item.name not in intruders:\n",
    "\n",
    "            annotation_video = annotations.read(item.path.split('_slice')[0] + '.csv')\n",
    "            \n",
    "            try:\n",
    "                copulation = annotation_video[0].events[0]\n",
    "\n",
    "                # Filter out videos where copulation is interrupted.\n",
    "                copulation_end = copulation.time_interval[1]\n",
    "                if copulation_end==N_FRAMES:\n",
    "                    print(item.name)\n",
    "                    continue\n",
    "\n",
    "                # Filter out videos where copulation lasts less than 8 minutes.\n",
    "                copulation_duration = copulation.duration\n",
    "                if copulation_duration <= 8 * 60 * FPS:\n",
    "                    print(item.name)\n",
    "                    continue\n",
    "            \n",
    "            except IndexError:\n",
    "                continue\n",
    "\n",
    "            experiments[condition].append(item.path.split('_slice')[0])\n",
    "\n",
    "experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:12.374528Z",
     "iopub.status.busy": "2021-09-10T00:52:12.373528Z",
     "iopub.status.idle": "2021-09-10T00:52:13.215051Z",
     "shell.execute_reply": "2021-09-10T00:52:13.214051Z",
     "shell.execute_reply.started": "2021-09-10T00:52:12.374528Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 0 \n",
      "Experiment: video_2017-09-05T13_28_41_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 1 \n",
      "Experiment: video_2017-09-05T14_34_55_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 2 \n",
      "Experiment: video_2017-09-06T13_42_13_arena1 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 3 \n",
      "Experiment: video_2017-09-06T14_45_32_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 4 \n",
      "Experiment: video_2017-09-07T14_37_38_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 5 \n",
      "Experiment: video_2017-09-07T14_37_38_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 6 \n",
      "Experiment: video_2017-09-08T13_26_13_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 7 \n",
      "Experiment: video_2017-09-12T13_30_06_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 8 \n",
      "Experiment: video_2017-09-12T14_36_55_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-09-12T14_36_55_arena3\n",
      "Iteration: 9 \n",
      "Experiment: video_2017-09-12T16_18_15_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 10 \n",
      "Experiment: video_2017-09-13T14_34_18_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 11 \n",
      "Experiment: video_2017-09-20T13_43_53_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-09-20T13_43_53_arena3\n",
      "Iteration: 12 \n",
      "Experiment: video_2017-09-20T14_48_55_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 13 \n",
      "Experiment: video_2017-09-21T13_38_55_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 14 \n",
      "Experiment: video_2017-09-21T15_57_23_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 15 \n",
      "Experiment: video_2017-09-26T13_40_19_arena1 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 16 \n",
      "Experiment: video_2017-09-28T13_36_06_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-09-28T13_36_06_arena4\n",
      "Iteration: 17 \n",
      "Experiment: video_2017-09-28T14_44_01_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 18 \n",
      "Experiment: video_2017-09-28T15_55_49_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-09-28T15_55_49_arena3\n",
      "Iteration: 19 \n",
      "Experiment: video_2017-10-12T15_50_43_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-10-12T15_50_43_arena2\n",
      "Iteration: 20 \n",
      "Experiment: video_2017-10-13T16_06_33_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-10-13T16_06_33_arena2\n",
      "Iteration: 21 \n",
      "Experiment: video_2017-10-18T15_19_17_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 22 \n",
      "Experiment: video_2017-10-19T13_50_56_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-10-19T13_50_56_arena2\n",
      "Iteration: 23 \n",
      "Experiment: video_2017-10-20T13_21_06_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 24 \n",
      "Experiment: video_2017-10-20T15_44_09_arena2 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 25 \n",
      "Experiment: video_2017-10-24T16_13_35_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 26 \n",
      "Experiment: video_2017-10-25T14_55_12_arena1 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 27 \n",
      "Experiment: video_2017-11-02T13_37_48_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 28 \n",
      "Experiment: video_2017-11-02T14_48_28_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 29 \n",
      "Experiment: video_2017-11-02T16_11_06_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 30 \n",
      "Experiment: video_2017-11-03T14_54_45_arena1 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "video_2017-11-03T14_54_45_arena1\n",
      "Iteration: 31 \n",
      "Experiment: video_2017-11-03T16_20_50_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 32 \n",
      "Experiment: video_2017-12-20T12_58_51_arena4 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 33 \n",
      "Experiment: video_2017-12-20T14_01_01_arena3 \n",
      "Condition: virgin_virgin \n",
      "\n",
      "Iteration: 0 \n",
      "Experiment: video_2017-09-05T13_28_41_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-09-05T13_28_41_arena3\n",
      "Iteration: 1 \n",
      "Experiment: video_2017-09-07T16_01_26_arena4 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-09-07T16_01_26_arena4\n",
      "Iteration: 2 \n",
      "Experiment: video_2017-09-08T14_31_59_arena2 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-09-08T14_31_59_arena2\n",
      "Iteration: 3 \n",
      "Experiment: video_2017-09-08T14_31_59_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 4 \n",
      "Experiment: video_2017-09-12T13_30_06_arena1 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 5 \n",
      "Experiment: video_2017-09-13T15_40_03_arena1 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 6 \n",
      "Experiment: video_2017-09-14T14_47_16_arena4 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-09-14T14_47_16_arena4\n",
      "Iteration: 7 \n",
      "Experiment: video_2017-09-20T13_43_53_arena1 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-09-20T13_43_53_arena1\n",
      "Iteration: 8 \n",
      "Experiment: video_2017-09-26T14_52_18_arena2 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 9 \n",
      "Experiment: video_2017-09-27T15_06_20_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-09-27T15_06_20_arena3\n",
      "Iteration: 10 \n",
      "Experiment: video_2017-10-11T13_30_05_arena1 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-11T13_30_05_arena1\n",
      "Iteration: 11 \n",
      "Experiment: video_2017-10-11T16_00_50_arena4 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-11T16_00_50_arena4\n",
      "Iteration: 12 \n",
      "Experiment: video_2017-10-13T14_37_35_arena1 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-13T14_37_35_arena1\n",
      "Iteration: 13 \n",
      "Experiment: video_2017-10-13T14_37_35_arena2 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-13T14_37_35_arena2\n",
      "Iteration: 14 \n",
      "Experiment: video_2017-10-13T14_37_35_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-13T14_37_35_arena3\n",
      "Iteration: 15 \n",
      "Experiment: video_2017-10-19T13_50_56_arena4 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-19T13_50_56_arena4\n",
      "Iteration: 16 \n",
      "Experiment: video_2017-10-19T14_58_20_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-19T14_58_20_arena3\n",
      "Iteration: 17 \n",
      "Experiment: video_2017-10-19T16_19_37_arena2 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 18 \n",
      "Experiment: video_2017-10-20T14_31_24_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 19 \n",
      "Experiment: video_2017-10-20T14_31_24_arena4 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-20T14_31_24_arena4\n",
      "Iteration: 20 \n",
      "Experiment: video_2017-10-24T13_43_26_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-24T13_43_26_arena3\n",
      "Iteration: 21 \n",
      "Experiment: video_2017-10-25T13_39_35_arena1 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 22 \n",
      "Experiment: video_2017-10-25T14_55_12_arena2 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 23 \n",
      "Experiment: video_2017-10-25T14_55_12_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-10-25T14_55_12_arena3\n",
      "Iteration: 24 \n",
      "Experiment: video_2017-10-26T13_35_36_arena2 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 25 \n",
      "Experiment: video_2017-10-26T14_43_16_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 26 \n",
      "Experiment: video_2017-11-03T13_38_06_arena2 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-11-03T13_38_06_arena2\n",
      "Iteration: 27 \n",
      "Experiment: video_2017-11-03T13_38_06_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "video_2017-11-03T13_38_06_arena3\n",
      "Iteration: 28 \n",
      "Experiment: video_2017-11-03T13_38_06_arena4 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 29 \n",
      "Experiment: video_2017-11-03T16_20_50_arena3 \n",
      "Condition: virgin_mated24h \n",
      "\n",
      "Iteration: 0 \n",
      "Experiment: video_2017-09-06T15_53_34_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-09-06T15_53_34_arena1\n",
      "Iteration: 1 \n",
      "Experiment: video_2017-09-07T16_01_26_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 2 \n",
      "Experiment: video_2017-09-07T16_01_26_arena2 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 3 \n",
      "Experiment: video_2017-09-08T15_42_18_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-09-08T15_42_18_arena1\n",
      "Iteration: 4 \n",
      "Experiment: video_2017-09-08T15_42_18_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 5 \n",
      "Experiment: video_2017-09-13T15_40_03_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 6 \n",
      "Experiment: video_2017-09-14T15_53_40_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-09-14T15_53_40_arena3\n",
      "Iteration: 7 \n",
      "Experiment: video_2017-09-20T16_07_55_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-09-20T16_07_55_arena4\n",
      "Iteration: 8 \n",
      "Experiment: video_2017-09-21T15_57_23_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 9 \n",
      "Experiment: video_2017-09-21T15_57_23_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 10 \n",
      "Experiment: video_2017-09-26T16_06_31_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-09-26T16_06_31_arena1\n",
      "Iteration: 11 \n",
      "Experiment: video_2017-09-26T16_06_31_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-09-26T16_06_31_arena3\n",
      "Iteration: 12 \n",
      "Experiment: video_2017-09-28T15_55_49_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-09-28T15_55_49_arena1\n",
      "Iteration: 13 \n",
      "Experiment: video_2017-10-11T16_00_50_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-10-11T16_00_50_arena1\n",
      "Iteration: 14 \n",
      "Experiment: video_2017-10-11T16_00_50_arena2 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-10-11T16_00_50_arena2\n",
      "Iteration: 15 \n",
      "Experiment: video_2017-10-11T16_00_50_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-10-11T16_00_50_arena3\n",
      "Iteration: 16 \n",
      "Experiment: video_2017-10-12T15_50_43_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-10-12T15_50_43_arena4\n",
      "Iteration: 17 \n",
      "Experiment: video_2017-10-13T16_06_33_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-10-13T16_06_33_arena4\n",
      "Iteration: 18 \n",
      "Experiment: video_2017-10-18T16_37_02_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-10-18T16_37_02_arena3\n",
      "Iteration: 19 \n",
      "Experiment: video_2017-10-18T16_37_02_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 20 \n",
      "Experiment: video_2017-10-19T16_19_37_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 21 \n",
      "Experiment: video_2017-10-19T16_19_37_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 22 \n",
      "Experiment: video_2017-10-25T16_23_26_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 23 \n",
      "Experiment: video_2017-10-25T16_23_26_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-10-25T16_23_26_arena3\n",
      "Iteration: 24 \n",
      "Experiment: video_2017-11-02T16_11_06_arena4 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "video_2017-11-02T16_11_06_arena4\n",
      "Iteration: 25 \n",
      "Experiment: video_2017-12-19T15_13_55_arena2 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 26 \n",
      "Experiment: video_2017-12-19T15_13_55_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 27 \n",
      "Experiment: video_2017-12-19T16_23_33_arena1 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 28 \n",
      "Experiment: video_2017-12-19T16_23_33_arena2 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 29 \n",
      "Experiment: video_2017-12-19T16_23_33_arena3 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 30 \n",
      "Experiment: video_2017-12-20T15_08_53_arena2 \n",
      "Condition: virgin_mated2h \n",
      "\n",
      "Iteration: 0 \n",
      "Experiment: video_2017-09-05T13_28_41_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 1 \n",
      "Experiment: video_2017-09-05T13_28_41_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 2 \n",
      "Experiment: video_2017-09-06T13_42_13_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-09-06T13_42_13_arena2\n",
      "Iteration: 3 \n",
      "Experiment: video_2017-09-06T14_45_32_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 4 \n",
      "Experiment: video_2017-09-06T14_45_32_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 5 \n",
      "Experiment: video_2017-09-08T13_26_13_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 6 \n",
      "Experiment: video_2017-09-08T13_26_13_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 7 \n",
      "Experiment: video_2017-09-12T14_36_55_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 8 \n",
      "Experiment: video_2017-09-12T16_18_15_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 9 \n",
      "Experiment: video_2017-09-12T16_18_15_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 10 \n",
      "Experiment: video_2017-09-13T13_31_57_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 11 \n",
      "Experiment: video_2017-09-20T16_07_55_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 12 \n",
      "Experiment: video_2017-09-20T16_07_55_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 13 \n",
      "Experiment: video_2017-09-21T13_38_55_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-09-21T13_38_55_arena2\n",
      "Iteration: 14 \n",
      "Experiment: video_2017-09-21T14_47_10_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-09-21T14_47_10_arena1\n",
      "Iteration: 15 \n",
      "Experiment: video_2017-09-26T16_06_31_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-09-26T16_06_31_arena2\n",
      "Iteration: 16 \n",
      "Experiment: video_2017-10-11T14_44_35_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 17 \n",
      "Experiment: video_2017-10-11T14_44_35_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-10-11T14_44_35_arena2\n",
      "Iteration: 18 \n",
      "Experiment: video_2017-10-12T13_31_34_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 19 \n",
      "Experiment: video_2017-10-13T13_32_39_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-10-13T13_32_39_arena1\n",
      "Iteration: 20 \n",
      "Experiment: video_2017-10-18T16_37_02_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-10-18T16_37_02_arena1\n",
      "Iteration: 21 \n",
      "Experiment: video_2017-10-18T16_37_02_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 22 \n",
      "Experiment: video_2017-10-19T13_50_56_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-10-19T13_50_56_arena1\n",
      "Iteration: 23 \n",
      "Experiment: video_2017-10-20T14_31_24_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-10-20T14_31_24_arena1\n",
      "Iteration: 24 \n",
      "Experiment: video_2017-10-20T14_31_24_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 25 \n",
      "Experiment: video_2017-10-26T14_43_16_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "Iteration: 26 \n",
      "Experiment: video_2017-10-26T16_04_34_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-10-26T16_04_34_arena1\n",
      "Iteration: 27 \n",
      "Experiment: video_2017-10-26T16_04_34_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-10-26T16_04_34_arena2\n",
      "Iteration: 28 \n",
      "Experiment: video_2017-11-02T14_48_28_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-11-02T14_48_28_arena1\n",
      "Iteration: 29 \n",
      "Experiment: video_2017-11-02T14_48_28_arena2 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-11-02T14_48_28_arena2\n",
      "Iteration: 30 \n",
      "Experiment: video_2017-12-20T12_58_51_arena1 \n",
      "Condition: virgin_added \n",
      "\n",
      "video_2017-12-20T12_58_51_arena1\n",
      "Iteration: 31 \n",
      "Experiment: video_2017-12-20T14_01_01_arena2 \n",
      "Condition: virgin_added \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>has_aggression</th>\n",
       "      <th>latency</th>\n",
       "      <th>latency_mins</th>\n",
       "      <th>nframes</th>\n",
       "      <th>nbouts</th>\n",
       "      <th>ratio_frames</th>\n",
       "      <th>ratio_bouts</th>\n",
       "      <th>second_female</th>\n",
       "      <th>second_fly_delay</th>\n",
       "      <th>condition</th>\n",
       "      <th>experiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>31444.0</td>\n",
       "      <td>8.734444</td>\n",
       "      <td>180</td>\n",
       "      <td>1</td>\n",
       "      <td>0.004277</td>\n",
       "      <td>0.085545</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>virgin_virgin</td>\n",
       "      <td>video_2017-09-05T13_28_41_arena4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>853.0</td>\n",
       "      <td>0.236944</td>\n",
       "      <td>617</td>\n",
       "      <td>10</td>\n",
       "      <td>0.011225</td>\n",
       "      <td>0.654950</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>virgin_virgin</td>\n",
       "      <td>video_2017-09-05T14_34_55_arena2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>11456.0</td>\n",
       "      <td>3.182222</td>\n",
       "      <td>343</td>\n",
       "      <td>6</td>\n",
       "      <td>0.009081</td>\n",
       "      <td>0.571867</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>virgin_virgin</td>\n",
       "      <td>video_2017-09-06T13_42_13_arena1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>True</td>\n",
       "      <td>3703.0</td>\n",
       "      <td>1.028611</td>\n",
       "      <td>950</td>\n",
       "      <td>13</td>\n",
       "      <td>0.018440</td>\n",
       "      <td>0.908403</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>virgin_virgin</td>\n",
       "      <td>video_2017-09-06T14_45_32_arena3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>True</td>\n",
       "      <td>12260.0</td>\n",
       "      <td>3.405556</td>\n",
       "      <td>579</td>\n",
       "      <td>7</td>\n",
       "      <td>0.012334</td>\n",
       "      <td>0.536798</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>virgin_virgin</td>\n",
       "      <td>video_2017-09-07T14_37_38_arena3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>6.361944</td>\n",
       "      <td>virgin_added</td>\n",
       "      <td>video_2017-10-26T16_04_34_arena1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0.611944</td>\n",
       "      <td>virgin_added</td>\n",
       "      <td>video_2017-10-26T16_04_34_arena2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>3.442222</td>\n",
       "      <td>virgin_added</td>\n",
       "      <td>video_2017-11-02T14_48_28_arena1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>5.978611</td>\n",
       "      <td>virgin_added</td>\n",
       "      <td>video_2017-12-20T12_58_51_arena1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>True</td>\n",
       "      <td>23537.0</td>\n",
       "      <td>6.538056</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000410</td>\n",
       "      <td>0.295070</td>\n",
       "      <td>82628</td>\n",
       "      <td>3.328056</td>\n",
       "      <td>virgin_added</td>\n",
       "      <td>video_2017-12-20T14_01_01_arena2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>124 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     has_aggression  latency  latency_mins  nframes  nbouts  ratio_frames  \\\n",
       "0              True  31444.0      8.734444      180       1      0.004277   \n",
       "1              True    853.0      0.236944      617      10      0.011225   \n",
       "2              True  11456.0      3.182222      343       6      0.009081   \n",
       "3              True   3703.0      1.028611      950      13      0.018440   \n",
       "4              True  12260.0      3.405556      579       7      0.012334   \n",
       "..              ...      ...           ...      ...     ...           ...   \n",
       "121           False      NaN      0.000000        0       0      0.000000   \n",
       "122           False      NaN      0.000000        0       0      0.000000   \n",
       "123           False      NaN      0.000000        0       0      0.000000   \n",
       "125           False      NaN      0.000000        0       0      0.000000   \n",
       "126            True  23537.0      6.538056       10       2      0.000410   \n",
       "\n",
       "     ratio_bouts  second_female  second_fly_delay      condition  \\\n",
       "0       0.085545              0          0.000000  virgin_virgin   \n",
       "1       0.654950              0          0.000000  virgin_virgin   \n",
       "2       0.571867              0          0.000000  virgin_virgin   \n",
       "3       0.908403              0          0.000000  virgin_virgin   \n",
       "4       0.536798              0          0.000000  virgin_virgin   \n",
       "..           ...            ...               ...            ...   \n",
       "121     0.000000              0          6.361944   virgin_added   \n",
       "122     0.000000              0          0.611944   virgin_added   \n",
       "123     0.000000              0          3.442222   virgin_added   \n",
       "125     0.000000              0          5.978611   virgin_added   \n",
       "126     0.295070          82628          3.328056   virgin_added   \n",
       "\n",
       "                           experiment  \n",
       "0    video_2017-09-05T13_28_41_arena4  \n",
       "1    video_2017-09-05T14_34_55_arena2  \n",
       "2    video_2017-09-06T13_42_13_arena1  \n",
       "3    video_2017-09-06T14_45_32_arena3  \n",
       "4    video_2017-09-07T14_37_38_arena3  \n",
       "..                                ...  \n",
       "121  video_2017-10-26T16_04_34_arena1  \n",
       "122  video_2017-10-26T16_04_34_arena2  \n",
       "123  video_2017-11-02T14_48_28_arena1  \n",
       "125  video_2017-12-20T12_58_51_arena1  \n",
       "126  video_2017-12-20T14_01_01_arena2  \n",
       "\n",
       "[124 rows x 11 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 817 ms\n"
     ]
    }
   ],
   "source": [
    "aggression_df = pd.DataFrame()\n",
    "\n",
    "for condition in condition_order:\n",
    "    for h, experiment in enumerate(experiments[condition]):\n",
    "\n",
    "        experiment_path = experiment\n",
    "        experiment_name = os.path.basename(experiment)\n",
    "        \n",
    "        print('Iteration:', h, '\\nExperiment:', experiment_name, '\\nCondition:', condition, '\\n')\n",
    "        \n",
    "        annotation_video = annotations.read(experiment_path + '.csv')\n",
    "\n",
    "        try:\n",
    "            copulation = annotation_video[0].events[0]\n",
    "            \n",
    "            # Exclude copulations that are not captured in their entirety.\n",
    "            copulation_end = copulation.time_interval[1]\n",
    "            if copulation_end==N_FRAMES:\n",
    "                continue\n",
    "            \n",
    "            # Exclude copulations whose duration is less than the supposed ejaculation time (8 minutes).\n",
    "            copulation_duration = copulation.duration\n",
    "            if copulation_duration <= 8 * 60 * FPS:\n",
    "                continue\n",
    "            \n",
    "        # Exclude experiments without copulation (since aggression is only observed during copulation).\n",
    "        except IndexError:\n",
    "            continue\n",
    "        \n",
    "        aggression_events = annotation_video[1].events\n",
    "        n_events = len(aggression_events)\n",
    "\n",
    "        # In case there is aggression, do the necessary calculations.\n",
    "        if n_events > 0:\n",
    "\n",
    "            new_fly = 0\n",
    "            latency =  aggression_events[0].time - copulation.time\n",
    "            latency_mins = latency / (60 * FPS)\n",
    "            ratio_frames = sum([aggression.duration for aggression in aggression_events]) / copulation.duration\n",
    "            ratio_bouts = n_events / (copulation.duration / (60 * FPS))\n",
    "            delay = 0\n",
    "\n",
    "            aggression_data = pd.DataFrame({'has_aggression': True,\n",
    "                                            'latency': latency,\n",
    "                                            'latency_mins': latency_mins,\n",
    "                                            'nframes': sum([aggression.duration for aggression in aggression_events]),\n",
    "                                            'nbouts': n_events,\n",
    "                                            'ratio_frames': ratio_frames,\n",
    "                                            'ratio_bouts': ratio_bouts,\n",
    "                                            'second_female': new_fly,\n",
    "                                            'second_fly_delay': delay,\n",
    "                                            'condition': condition,\n",
    "                                            'experiment': experiment_name},\n",
    "                                            index=[h],\n",
    "                                           )\n",
    "\n",
    "        # If not, preset our dictionary with default values.\n",
    "        else:\n",
    "            print(experiment_name)\n",
    "            delay = 0\n",
    "\n",
    "            aggression_data = pd.DataFrame({'has_aggression': False,\n",
    "                                            'latency': np.nan,\n",
    "                                            'latency_mins': 0,\n",
    "                                            'nframes': 0,\n",
    "                                            'nbouts': 0,\n",
    "                                            'ratio_frames': 0, \n",
    "                                            'ratio_bouts': 0,\n",
    "                                            'second_female': 0,\n",
    "                                            'second_fly_delay': delay,\n",
    "                                            'condition': condition,\n",
    "                                            'experiment': experiment_name},\n",
    "                                            index=[h],\n",
    "                                           )\n",
    "\n",
    "        # Concatenate all data together.\n",
    "        aggression_df = pd.concat([aggression_df, aggression_data], ignore_index=True)\n",
    "\n",
    "# Apply a filter to discard all experiments we don't care about.\n",
    "aggression_df = aggression_df.query('second_fly_delay<8').dropna(subset=['ratio_frames'])\n",
    "aggression_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Outliers Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:13.217035Z",
     "iopub.status.busy": "2021-09-10T00:52:13.216051Z",
     "iopub.status.idle": "2021-09-10T00:52:13.246019Z",
     "shell.execute_reply": "2021-09-10T00:52:13.245034Z",
     "shell.execute_reply.started": "2021-09-10T00:52:13.217035Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(list, {'virgin_added': ['video_2017-10-12T13_31_34_arena1']})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time: 21 ms\n"
     ]
    }
   ],
   "source": [
    "outlier_experiments = defaultdict(list)\n",
    "\n",
    "for condition in condition_order:\n",
    "\n",
    "    # Fetch the relevant column to filter by, which for us will be the \"ratio_frames\" columns.\n",
    "    dataset = aggression_df.query('condition==\"' + condition + '\"')['ratio_frames']\n",
    "\n",
    "    # Get the index for each outlier.\n",
    "    outlier_positions = helpers.check_outliers(dataset)\n",
    "\n",
    "    for outlier in outlier_positions:\n",
    "    \n",
    "        # For each outlier index found previously, get the corresponding experiment name.\n",
    "        experiment = aggression_df.loc[outlier, ['experiment']].values[0]\n",
    "\n",
    "        # Store the outlier experiment into the dictionary, associate to its respective condition.\n",
    "        outlier_experiments[condition].append(experiment)\n",
    "    \n",
    "outlier_experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Writing to File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-10T00:52:13.248019Z",
     "iopub.status.busy": "2021-09-10T00:52:13.247019Z",
     "iopub.status.idle": "2021-09-10T00:52:13.278002Z",
     "shell.execute_reply": "2021-09-10T00:52:13.277003Z",
     "shell.execute_reply.started": "2021-09-10T00:52:13.247019Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t virgin_added\n",
      "Outliers already detected!\n",
      "\n",
      "Done!\n",
      "\n",
      "time: 8.99 ms\n"
     ]
    }
   ],
   "source": [
    "for condition in outlier_experiments.keys():\n",
    "    \n",
    "    print('\\t', condition)\n",
    "\n",
    "    # Path to currently existing \"quality_control\" file.\n",
    "    path_to_quality_file = os.path.join(EXPERIMENTS_PATH, condition)\n",
    "\n",
    "    for item in os.scandir(path_to_quality_file):\n",
    "\n",
    "        if item.name == 'quality_control.csv':\n",
    "            \n",
    "            # If there are already \"outlier\" comments in the csv, then we've already detected outliers and we can skip this part.\n",
    "            quality_df = pd.read_csv(item.path, usecols=[0,1,2])\n",
    "            if 'outlier' in quality_df['comment'].values:\n",
    "                print('Outliers already detected!')\n",
    "                break\n",
    "\n",
    "            # Get outliers for the current condition.\n",
    "            current_outliers = outlier_experiments[condition]\n",
    "            print(current_outliers, '\\n')\n",
    "\n",
    "            # Create a new, blank temporary file.\n",
    "            temp_file = os.path.join(path_to_quality_file, 'temp.csv')\n",
    "\n",
    "            with open(item, mode='r') as input_file, open(temp_file, mode='w') as output_file:\n",
    "\n",
    "                # Read existing \"quality_control\" file line by line.\n",
    "                for line in input_file.readlines():\n",
    "\n",
    "                    is_edited = False\n",
    "                    for outlier in current_outliers:\n",
    "\n",
    "                        if outlier in line:\n",
    "\n",
    "                            # Edit the existing line where an outlier was found.\n",
    "                            new_line = line.replace('TRUE', 'FALSE,outlier')\n",
    "\n",
    "                            # Write the edited line to the temporary file.\n",
    "                            output_file.write(new_line)\n",
    "\n",
    "                            is_edited = True\n",
    "\n",
    "                    if is_edited == False:\n",
    "\n",
    "                        # Write any unedited lines as they are to the temporary file.\n",
    "                        output_file.write(line)\n",
    "\n",
    "            # Delete the old, pre-existing \"quality_control\"\n",
    "            os.remove(item)\n",
    "            \n",
    "            # Rename the temporary file to \"quality_control\".\n",
    "            os.rename(temp_file, os.path.join(path_to_quality_file, 'quality_control.csv'))\n",
    "            \n",
    "print('\\nDone!\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
